#!/usr/bin/env python
#coding:utf-8
# Author:  GreySign 
# Purpose: 正则匹配工具包
# Created: 2011/03/13
# Update:  2012/3/13
VERSION = 0.2
# History:
"""
2012/3/13: 
           1.修改获取标签正文的接口，改为获取所有标签正文返回列表
2012/3/15: 
           1.分类规划为：正则自动生成、ascii编码处理、实体符编码处理、HTML标签处理、格式校验、文本提取、代码混淆处理
           2.新增ascii编码处理能力，可处理8、10、16进制ascii编码、escape编码自动递归多次解密
           3.新增实体符编码处理能力，可处理16进制、10进制的编码实体符转换
           4.新增自动生成长正则能力
           5.新增代码混淆处理能力，可处理字符串拼接问题

"""
import re
import trie,decoder



__all__ = ['www']


########################################################################
class FormatError(Exception):
	""""""
	pass


########################################################################
class WWW(object):
	"""
	关于WEB的那些匹配处理和格式化处理
	1.is_xxxx系列：做格式验证
	2.get_xxxx系列：获取特定文本
	"""


	#----------------------------------------------------------------------
	def __init__(self):
		"""Constructor"""
		self.is_funcs = [
			self.is_domain_format,
			self.is_ip_format,
		    self.is_linux_path_format
		]
		self.get_funcs = [
			self.get_domain,
			self.get_rootdomain,
			self.get_tag_att_value,
			self.get_tag_content
		]
		self.trie = trie.Trie()
		self.decoder = decoder.SmartDecoder()

	####################
	##正则自动生成相关
	####################
	#----------------------------------------------------------------------
	def give_me_regex(self,strings):
		"""
		自动寻找可匹配该字符串的格式验证规则
		in: 字符串
		out: 可匹配成功的规则
		"""
		if not strings:
			return {}
		regexs = {}
		for func in self.is_funcs:
			if func(strings):
				print '[*]`%s` Matched:'%strings,func.func_name
				print getattr(self,func.func_name+'_reg')
				regexs.update( {func:getattr(self,func.func_name+'_reg')})
		return regexs

	#----------------------------------------------------------------------
	def generation_string_regex(self,strings):
		"""
		基于大量格式类似字符串生成一个正则，用于加速匹配用，不用变量大量字符串进行匹配，一个规则搞定
		in : ['foobar', 'foobah']
		out : fooba[rh]
		"""
		self.trie.data = {}
		for w in strings:    #like 'fooxar', 'foozap', 'fooza']:
			self.trie.add(w)
		return self.trie.regexp()

	####################
	##处理字符ascii编码
	####################
	#----------------------------------------------------------------------
	def ascii_decode(self,html):
		"""
		in：html字符串
		out：调用decoder进行各种解码处理，返回解码后的html
		"""
		return self.decoder.decode(html)

	####################
	##处理实体符编码相关
	####################
	def _get_unicode_char(self,char,charset='utf8'):
		""""""
		try:
			char = char.decode(charset)
			s = ''
			s += str(hex(ord(char) >> 8)).replace('0x','')
			s += str(hex(ord(char) & 0xff)).replace('0x','')
			#print repr(s)
			if len(s)==3:
				s= '0'+s
		except:
			return False
		return s


	#----------------------------------------------------------------------
	def gen_html_hex_char(self,char):
		"""
		生成16进制的html实体符
		"""
		unicode_char = self._get_unicode_char(char)
		if not unicode_char:
			return False
		return '&#x' + unicode_char

	#----------------------------------------------------------------------
	def gen_html_decimal_char(self,char):
		"""
		生成10进制的html实体符
		"""
		if not char.isalnum():
			return False
		return '&#' + str(ord(char))



	#----------------------------------------------------------------------
	def _conv_html_decimal_char(self,html_char):
		"""
		对10进制的HTML实体符进行解码
		"""
		html_char = html_char.replace('&#','')
		return chr(int(html_char.lstrip('0')))

	#----------------------------------------------------------------------
	def _conv_html_hex_char(self,html_char):
		"""
		对16进制的HTML实体符进行解码
		"""
		try:
			html_char = html_char.replace('&#x','')
			if not html_char.isalnum():
				return False
			exec("char = u'\u%s'"%html_char)
			char = char.encode('utf8')
		except:
			return False
		return char

	#----------------------------------------------------------------------
	def conv_html_char(self,html_char):
		"""
		对HTML实体符进行解码，支持10、16进制的实体符
		"""
		if html_char.startswith('&#x'):
			return self._conv_html_hex_char(html_char)
		elif html_char.startswith('&#'):
			return self._conv_html_decimal_char(html_char)
		return False        



	####################
	##HTML标签处理
	####################
	def get_tag_att_value(self,html,tag):
		"""获取指定tag的属性名与值的字典集合列表
		in：
			"<img src=# onerror=alert(/[code]/) /><img src=# onerror=alert(/[code2]/) />"
		out：
			[{'onerror': 'alert(/[code]/)', 'src': '#'}, {'onerror': 'alert(/[code2]/)', 'src': '#'}]
		"""
		values = []
		def get_tag_value_content():
			tag_value_content_pattern = r"""<%s([^>]+)>"""%tag
			tag_value_contents = re.findall(tag_value_content_pattern,html,re.I)
			return tag_value_contents

		def get_att_name():
			all_att_names = []
			att_pattern = r"""\s\b([\w]+?)\s*="""
			att_pattern_obj = re.compile(att_pattern,re.I)
			for value_content in value_contents:
				if value_content:
					att_names = att_pattern_obj.findall(value_content)
					if att_names:
						all_att_names.append((value_content,att_names))
			return all_att_names

		def get_att_value():
			att_values = []
			for value_content,att_names in all_att_names:
				att_name_value = {}
				for att_name in att_names:
					tag_value_pattern = r"""
									  \b%s\s*=\s* #属性及赋值符号
									  (?:[\\]?"([^"]*?)[\\]?" #双引号情况，或者是……
									  |
									  [\\]?'([^']*?)[\\]?' #单引号情况，或者是……
									  |
									  ([^>'"\s]+) #无引号情况
									  )"""%att_name
					values = re.findall(tag_value_pattern,value_content,re.I+re.X)
					if values == []:
						continue
					for value in values[0]:
						if not value:
							continue
						if value != "" :
							att_name_value[att_name] = value
				att_values.append(att_name_value)
			return att_values
		value_contents = get_tag_value_content()
		all_att_names = get_att_name()
		att_values = get_att_value()
		return att_values

	def get_tag_content(self,html,tag_name):
		'''
		获取指定标签正文
		in : <script>alert(/[code]/)</script>
		out: ['alert(/[code]/)']
		'''
		if '<%s'%tag_name not in html:
			return []
		regex = r"""<%(tag_name)s[^>]*?>(.+?)</%(tag_name)s>"""%vars()
		compile_obj = re.compile(regex,  re.IGNORECASE|re.DOTALL)
		contents = compile_obj.findall(html)
		#if not contents:return []
		'''
		try:
			index = int(index)
			if index < 1:
				index = 1
		except:
			index = 1
		try:
			result = contents[index-1]
		except:
			result = contents[0]
		'''
		return contents

	####################
	##格式校验
	####################
	#----------------------------------------------------------------------
	def is_ip_format(self,ip_str):
		"""
		检查IP是否符合格式规范
		"""
		if not hasattr(self,'is_ip_format_re_obj'):
			self.is_ip_format_reg = """
                        ^ #必须是串开始
                        (?:\d{1,2}|1\d\d|2[0-4]\d|25[0-5])
                        \.
                        (?:\d{1,2}|1\d\d|2[0-4]\d|25[0-5])
                        \.
                        (?:\d{1,2}|1\d\d|2[0-4]\d|25[0-5])
                        \.
                        (?:\d{1,2}|1\d\d|2[0-4]\d|25[0-5])
                        $		
                    """
			self.is_ip_format_re_obj = re.compile(self.is_ip_format_reg,re.X)
		if self.is_ip_format_re_obj.search(ip_str):
			return True
		else:
			return False

	#----------------------------------------------------------------------
	def is_domain_format(self,domain):
		"""
		一个完整的域名，由根域、顶级域、二级、三级……域名构成，每级域名之间用点分开，
		每级域名由字母、数字和减号构成（第一个字母不能是减号），不区分大小写，长度不超过63。
		"""
		if not hasattr(self,'is_domain_format_re_obj'):
			self.is_domain_format_reg = """^[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(?:\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})*?(?:\.[a-zA-Z]{1,62}[-a-zA-Z]{1,62})\.?$"""
			self.is_domain_format_re_obj = re.compile(self.is_domain_format_reg,re.M)
		if self.is_domain_format_re_obj.search(domain):
			return True
		else:
			return False

	#----------------------------------------------------------------------
	def is_linux_path_format(self,filepath):
		""""""
		if not hasattr(self,'is_linux_path_format_re_obj'):
			self.is_linux_path_format_reg = """^([\/][\w.-]+)*$"""
			self.is_linux_path_format_re_obj = re.compile(self.is_linux_path_format_reg,re.M)
		if self.is_linux_path_format_re_obj.search(filepath):
			return True
		else:
			return False
		

	####################
	##文本提取
	####################
	def get_domain(self, url):
		"""获取指定url的域名格式：www.knownsec.com"""
		url = url.lower()
		head_pos = url.find('//')
		if head_pos != -1:
			url = url[head_pos+2:]
		end_pos = url.find('/')
		if end_pos != -1:
			url = url[:end_pos]
		else:
			end_pos = url.find('?')
			if end_pos != -1:
				url = url[:end_pos]
			else:
				end_pos = url.find('#')
				if end_pos != -1:
					url = url[:end_pos]
		domain = url
		return domain

	def get_rootdomain(self,url):
		"""
		获取指定url的根域名格式：
		www.baidu.com ==> baidu.com
		www.baidu.com.cn ==> baidu.com.cn
		"""
		domain = self.get_domain(url).strip()
		if '%' in domain :
			return False
		domain_list = domain.split('.')
		if domain_list[-1].isdigit():
			return False
		for suffix in domain_suffixs_double:
			if domain.endswith(suffix):
				rootDoamin = '.'.join(domain_list[-3:])
				return rootDoamin
		rootDomain = '.'.join(domain_list[-2:])
		return rootDomain


	#----------------------------------------------------------------------
	def get_formated_url(self,url):
		"""
		获取带协议和目录符格式的url。
		示例：
		baidu.com ==> http://baidu.com/
		http://baidu.com ==> http://baidu.com/
		http://baidu.com/1.php ==> http://baidu.com/1.php
		baidu.com/123/ ==> http://baidu.com/123/
		baidu.com/1.php?123 ==> http://baidu.com/1.php?123
		"""
		if not url.startswith('http://') and not url.startswith('https://'):
			url = 'http://' + url
		if url.endswith('/'):
			return url
		if url.replace('://','').find('/') == -1:
			url += '/'
		return url    

	def get_split_params(self,url):
		'''
		in：
		'http://translate.google.cn/?hl=zh-CN&tab=wT#123'
		out：
		（'http://translate.google.cn/','hl=zh-CN&tab=wT#123')
		'''
		try:
			params_start = url.index('?')
		except ValueError:
			params_start =False
		if params_start:
			no_params_url = url[:params_start]
			params = url[params_start+1:]
		else:
			no_params_url = url
			params = ''
		return {'no_params_url':no_params_url,'params':params}
	
	####################
	##代码混淆还原处理
	####################
	#----------------------------------------------------------------------
	def fix_join_string(self,string):
		""""""
		#fix js字符串拼接
		string = re.sub(r"""["']\s*\+\s*["']""",'',string)
		#fix vbs字符串拼接
		string = re.sub(r"""['"]\s*&\s*['"]""",'',string)
		return string

	#----------------------------------------------------------------------
	def _test(self):
		""""""
		str1 = "<script>alert(/[code]/)</script>"
		str2 = "<img src=# onerror=alert(/[code]/) /><img src=# onerror=alert(/[code2]/) />"
		assert self.get_tag_att_value(str2,'img'),[{'onerror': 'alert(/[code]/)', 'src': '#'}, {'onerror': 'alert(/[code2]/)', 'src': '#'}]
		assert self.get_tag_content(str1,'script'),'alert(/[code]/)'
		if not self.is_domain_format('wcc.cc'):
			raise FormatError,'[!]domain format ERROR!'
		if not self.is_ip_format('10.10.1.1'):
			raise FormatError,'[!]ip format ERROR!'
		print '[*]test ok.'


www = WWW()
if __name__ == '__main__':
	#www._test()
	str2 = "<script>alert(/[code]/)</script><script>123</script>"
	print www.get_tag_content(str2,'script')    
	www.give_me_regex('8.8.8.8')
	www.give_me_regex('a.baidu.com')
	print www.generation_string_regex(['foobah','foobar'])    
	data = '%3C%73%63%72%69%70%74%20%6C%61%6E%67%75%61%67%65%3D%76%62%73%63%72%69%70%74%3E'
	print www.ascii_decode(data)    
	print www._get_unicode_char('a')
	print www.gen_html_decimal_char('a')
	print www.gen_html_hex_char('测')
	print www.conv_html_char('&#x6d4b')    
	flower_code = """document.write('<d' + 'iv st' + 'yle' + '="po' + 'si' + 'tio' + 'n:a' + 'bso' + 'lu' + 'te;l' + 'ef' + 't:' + '-' + '11' + '20' + '3' + 'p' + 'x;' + '"' + '>');"""
	print www.fix_join_string(flower_code)
	print www.is_linux_path_format('/var/fdsa')